package com.ustadzmenjawab.crawler.en;

import java.io.IOException;
import java.util.Set;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.google.gwt.dev.util.collect.HashSet;
import com.ustadzmenjawab.crawler.CrawlerRoot;
import com.ustadzmenjawab.shared.Article;

public class OnIslamCrawler extends CrawlerRoot {
	public static final String HOME_URL = "http://www.onislam.net/english/";
	public static final String PATTERN = "onislam.net/english/";
	@Override
	public boolean isRelevantURL(String url) {
		if(url.contains("#"))
			return false;
		if(url.contains(".html")&&(url.contains("ask-the-scholar")
				|| url.contains("ask-about-hajj-and-umrah")
				|| url.contains("ask-about-islam")
				|| url.contains("ask-the-counselor")
				|| url.contains("ask-about-parenting"))
		)
			return true;
		return false;
	}

	@Override
	public boolean isRelevantPage(String content) {
		if(content.contains("Tanya") && content.contains("Jawab")){
			return true;
		}
		return false;
	}

	public boolean isRelevantPage(String url, String content){
		if(url.endsWith(".html") && (url.contains("/ask-the-scholar/")
				|| url.contains("/ask-about-hajj-and-umrah/")
				|| url.contains("/ask-about-islam/")
				|| url.contains("/ask-the-counselor/")
				|| url.contains("/ask-about-parenting/"))
		){
			return true;
		}
		return false;
	}

	@Override
	public String getTitle(String body) {
		Document doc = Jsoup.parse(body);
		Element p = doc.select("h2[class*=contentheading]").first();
		return p.text();
	}

	@Override
	public String getQuestion(String body) {
		Document doc = Jsoup.parse(body);
		//body =getArticle(body);
		Element p = doc.select("td[class*=cck_field editorquestion]").first();
		String text = p.text();
		return text;
	}

	@Override
	public String getArticle(String body) {
		Document doc = Jsoup.parse(body);
		Element p = doc.select("table[class*=admintable]").first();
		String article = p.text();
		return article;
	}

	@Override
	public boolean insertContent(String url, String content) {
		try{
			Article article = new Article();
			String title = getTitle(content);
			if(isTitleExist(title, "onislam.net")){
				return false;
			}
			article.setJudul(title);
			article.setKategori("shariah");
			article.setSumber("onislam.net");
			article.setTahun("2011");
			article.setUrl(url.replace("208.43.71.196", "www.onislam.net"));
			article.setUstadz("OnIslam Consultant");
			article.setArtikel(getArticle(content));
			article.setPertanyaan(getQuestion(content));
			db.insertArticle(article);
			return true;
		}catch(Exception e){
			System.out.println(" exception "+e.getClass() +" :" + e.getMessage());
			e.printStackTrace();
			insertFailure(url,"exception "+e.getClass() +" :" + e.getMessage());
			return false;
		}
	}

	public static void main(String[] args){
		try {
			OnIslamCrawler cr = new OnIslamCrawler();
			Document doc = Jsoup.connect(HOME_URL).timeout(10 * 60 * 1000).get();
			Elements unvisited = doc.select("a[href]");
			Set<String> visited = new HashSet<String>();
			while(unvisited.size() > 0){
				Element link = unvisited.first();
				try{
					String absHref = link.attr("abs:href");
					unvisited.remove(link);
					if(!visited.contains(absHref.toLowerCase())){
						visited.add(absHref.toLowerCase());
						if(cr.isRelevantURL(absHref)){
							Document page = Jsoup.connect(absHref).timeout(10 * 60 * 1000).get();
							String content = page.html();
							Elements pageLinks = page.select("a[href]");
							for(Element newlink : pageLinks){
								String newLinkStr = newlink.attr("abs:href").toLowerCase();
								if(!visited.contains(newLinkStr) && cr.isRelevantURL(newLinkStr))
								{
									unvisited.add(newlink);
								}
							}
							if(cr.isRelevantPage(absHref,content)){
								System.out.println(absHref + " is a relevant page");

								if(!cr.isUrlExist(absHref)){
									cr.insertContent(absHref, content);
								}
							}else{
								System.out.println(absHref + " is not a relevant page");
							}
						}
					}
				}catch(Exception e){
					System.out.println(" exception "+e.getClass() +" :" + e.getMessage());
					e.printStackTrace();
				}
			}
		}
		catch (IOException e) {
			e.printStackTrace();
		}

	}
}
